from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
from os import walk

print "Begins..."
# 1000, 100 and 10
TOPK = 10
print "TOPK:",TOPK

trecIDAndHitsDict = {}
f = []

# for 5K testing AND
# f.append("rawResults_TOP1000_AND_testingQueries_20140827")
# mypath = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/"
# for 5K testing OR
# f.append("rawResults_50%_TOP1000_OR_20140126Night")
# mypath = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/queryProcessingCostAnalysis/gov2/"

'''
# for 95K AND
f.append("rawResults_100%_TOP1000_AND_20140101Night_xaa")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xab")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xac")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xad")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xae")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xaf_Part1")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xaf_Part2")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xag_Part1")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xag_Part2")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xah_Part1")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xah_Part2")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xai_Part1")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xai_Part2")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xaj_Part1")
f.append("rawResults_100%_TOP1000_AND_20140101Night_xaj_Part2")
mypath = "/home/diaosi/workspace/web-search-engine-wei-2014-April/results/"
'''

'''
# for 95K OR
f.append("xaa_head95K_OR_rawResult_20140828")
f.append("xab_head95K_OR_rawResult_20140828")
f.append("xac_head95K_OR_rawResult_20140828")
f.append("xad_head95K_OR_rawResult_20140828")
f.append("xae_head95K_OR_rawResult_20140828")
f.append("xaf_head95K_OR_rawResult_20140828")
f.append("xag_head95K_OR_rawResult_20140828")
f.append("xah_head95K_OR_rawResult_20140828")
f.append("xai_head95K_OR_rawResult_20140828")
f.append("xaj_head95K_OR_rawResult_20140828")
f.append("xak_head95K_OR_rawResult_20140828")
f.append("xal_head95K_OR_rawResult_20140828")
f.append("xam_head95K_OR_rawResult_20140828")
f.append("xan_head95K_OR_rawResult_20140828")
f.append("xao_head95K_OR_rawResult_20140828")
f.append("xap_head95K_OR_rawResult_20140828")
f.append("xaq_head95K_OR_rawResult_20140828")
f.append("xar_head95K_OR_rawResult_20140828")
f.append("xas_head95K_OR_rawResult_20140828")
mypath = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/fakeQueryLogRelated/or_semantics_results/95KTrainingQueries/"
'''

# 0.1M fake queries AND
mypath = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/fakeQueryLogRelated/and_semantics_results/dot1MFakeQueries/"
# 9.9M fake queries AND
# @home
# mypath = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/fakeQueryLogRelated/and_semantics_results/9Dot9MFakeQueries/"
# @local
# mypath = "/local_scratch/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/fakeQueryLogRelated/and_semantics_results/9Dot9MFakeQueries/"

'''
# 0.1M fake queries OR
# mypath = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/fakeQueryLogRelated/or_semantics_results/dot1MFakeQueries/"
# 9.9M fake queries OR
mypath = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/fakeQueryLogRelated/or_semantics_results/9Dot9MFakeQueries/"
'''
for (dirpath, dirnames, filenames) in walk(mypath):
    f.extend(filenames)
    break
print len(f)

completePath = ""
ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/0.1MFakeQueries_AND_TOP10_docHitRanks_20140902"
ofh = open(ofn,"w")
print "ofn:",ofn

for fileName in f:
    completePath = mypath + fileName
    ifn = completePath
    ifh = open(ifn,"r")
    print "ifn:",ifn
    l = ifh.readline()
    while l:
        le = l.strip().split(" ")
        if len(le) == 25 and le[-2].startswith("GX"):
            theRank = int(le[0])
            if theRank <= TOPK:
                currTrecID = le[-2]
                # print currTrecID,theRank
                if currTrecID not in trecIDAndHitsDict:
                    trecIDAndHitsDict[currTrecID] = []
                    if len(trecIDAndHitsDict) % 100000 == 0:
                        print "len(trecIDAndHitsDict):",len(trecIDAndHitsDict)
                        # break
                trecIDAndHitsDict[currTrecID].append(theRank)

            else:
                pass
        l = ifh.readline()

print "Overall:"
print "ofn:",ofn
for currTrecID in trecIDAndHitsDict:
    outputLine = currTrecID + " "
    outputLine += str( len(trecIDAndHitsDict[currTrecID]) ) + " "
    for theRank in trecIDAndHitsDict[currTrecID]:
        outputLine += str(theRank) + " "
    outputLine = outputLine.strip()
    ofh.write(outputLine + "\n") 
ifh.close()
ofh.close()
print "Ends."





